#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import time
import sys
from pybloom_live import ScalableBloomFilter

reload(sys)
sys.setdefaultencoding('utf-8')



file_name = 'bidding_history'
index = 0
mobile_bloom = ScalableBloomFilter(initial_capacity=1000000, error_rate=0.001, mode=2)  # 慢点但是省内存
phone_bloom = ScalableBloomFilter(initial_capacity=1000000, error_rate=0.001, mode=2)  # 慢点但是省内存
email_bloom = ScalableBloomFilter(initial_capacity=1000000, error_rate=0.001, mode=2)  # 慢点但是省内存
other_bloom = ScalableBloomFilter(initial_capacity=1000000, error_rate=0.001, mode=2)  # 慢点但是省内存
with open(file_name,'r' ) as f ,open('tj_result', 'w'  ) as log_f:
    ls = []
    result_tj = {
        '':0,
    }
    for line in f:
        line = line.strip()
        if line:
            ls.append(line)
            dic = json.loads(line)
            contact_ls = dic.get('biddingProjectContact')
            if contact_ls:
                for tmp_contact_dic in contact_ls:
                    contact = tmp_contact_dic.get('contact','')
                    contactType = tmp_contact_dic.get('contactType','')
                    _id = "{}_{}".format(contact,contactType)
                    if contactType == 0:
                        mobile_bloom.add(_id)
                    elif contactType == 1:
                        phone_bloom.add(_id)
                    elif contactType == 2:
                        email_bloom.add(_id)
                    else:
                        other_bloom.add(_id)


        if len(ls) == 1000:
            log_name = generate_venom_name()
            with open(log_name,'w' ) as f:
                f.write('\n'.join(ls))
                ls = []
            print('当前bloom长度为：{}'.format(mobile_bloom.count))
            log_f.write('{}\t{}\t{}\t{}\n'.format(mobile_bloom.count,phone_bloom.count,email_bloom.count,other_bloom.count))
    else:
        if len(ls):
            log_name = generate_venom_name()
            with open(log_name, 'w' ) as f:
                f.write('\n'.join(ls))
            log_f.write(
                '{}\t{}\t{}\t{}\n'.format(mobile_bloom.count, phone_bloom.count, email_bloom.count, other_bloom.count))


